from feature_data.XmlData2DictData import Xml2Dict
from feature_data.const_data import *
from nd_utils.venue_re_util import *

xml_file_name_list = [
        xml_file_name1,
        xml_file_name2,
        xml_file_name3,
        xml_file_name4,
        xml_file_name5,
        xml_file_name6,
        xml_file_name7,
        xml_file_name8,
        xml_file_name9,
        xml_file_name10
    ]
paper_venue_set = set()

for xml_file_name in xml_file_name_list:

    xml2dict = Xml2Dict(xml_dir + xml_file_name)

    publication_dict = xml2dict.publication_dict

    for paperid, publication_attr_dict in publication_dict.items():
        venue = publication_attr_dict['jconf']
        paper_venue_set.add(venue)

paper_venue_list = sorted(list(paper_venue_set))

from nd_utils.nltk_util import Nltk_util
nltk_util = Nltk_util()

venue_set = set()
i = 0
for venue in paper_venue_list:
    # print venue
    if len( venue.split(' ')) > 1:
        # venue= clean_text(venue)
        venue = clean_text_digit(venue)
        if len(venue.split(' ')) > 1:
            # print venue.split(' ')
            # venue = clean_main_bak2(venue)
            venue = clean_main(venue)
            i += 1
            venue_set.add( venue )
print len(venue_set), i

venue_list = list(venue_set)
venue_list = sorted(venue_list)
for venue in venue_list:
    # print nltk_util.filter_stop_words(venue)
    print venue
    # print "===="
    # print nltk_util.stem_process_venue(venue)


# from nd_utils.nltk_util import Nltk_util
# nltk_util = Nltk_util()
#
# for venue in paper_venue_list:
#     venue =  nltk_util.stem_process_venue( venue )
#     print venue

def get_clean_paper_venue_list(paper_venue_list):
    venue_set = set()
    for venue in paper_venue_list:
        if len(venue.split(' ')) > 1:
            venue = clean_text_digit(venue)
            if len(venue.split(' ')) > 1:
                venue = clean_main(venue)
                venue_set.add(venue)
            else:
                venue_set.add(venue)
        else:
            venue_set.add(venue)
